/**
 * 
 */
package com.qunar.crawler.worker;

import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.qunar.crawler.html.HtmlAction;
import com.qunar.crawler.html.HtmlPossesser117Go;
import com.qunar.crawler.utils.TourUrlFrontier;
import com.qunar.crawler.utils.UrlFrontier;

/**
 * @author tao.zhang
 */
public class TourCrawler extends Thread {

	private static final Logger logger = LoggerFactory
			.getLogger(TourCrawler.class);

	@Override
	public void run() {

		while (true) {

			// 旅行列表URL
			String tourListUrl = TourUrlFrontier.getInstance().take();
			if (StringUtils.isBlank(tourListUrl))
				break;
			// 旅行URL容器
			List<String> tourUrls = new ArrayList<String>();
			long start = System.currentTimeMillis();

			// 抓取旅行列表界面
			String tourlistPage = null;

			HtmlAction action = new HtmlAction();
			
			// loop craw until success
			while (StringUtils.isBlank(tourlistPage)) {

				tourlistPage = action.get(tourListUrl);
			}
			// 提取旅行url
			HtmlPossesser117Go.digTourUrls(tourlistPage, tourUrls);

			long spend = System.currentTimeMillis() - start;
			// for (String url : tourUrls) {
			// System.out.println(url);
			//
			// }
			// logger.info("本次共爬到" + tourUrls.size() + "个旅程;耗时：" + spend / 1000
			// + "s");

			/** ----url内存去重---- */
			if (CollectionUtils.isNotEmpty(tourUrls)) {
				Set<String> set = new HashSet(tourUrls);
				tourUrls.clear();
				tourUrls.addAll(set);
			}

			// logger.info("去重后，旅程个数为：" + tourUrls.size());

			// DBUtil.addTours2DB(tourUrls);
			UrlFrontier.getInstance().put(tourUrls);
			logger.info("抓取旅行…………");
			logger.info("--------新增<<" + tourUrls.size()
					+ ">>个旅行Url到队列----------");
		}
		logger.info("--------TourUrlFrontier队列抓取完毕队列中共<<"
				+ UrlFrontier.getInstance().size()
				+ ">>个URL "
				+ new SimpleDateFormat("yyyy-MM-dd:HH:mm:ss")
						.format(new Date()) + "-------");
	}

}
